/* global importScripts db performance searchPlaces postMessage fullTextPlacesSearch */
console.log('worker started ', performance.now());
// import Dexie from 'dexie'
// import {db} from '../util/database.js'
// import {fullTextPlacesSearch,tokenize} from './fullTextSearch.js'
import 'string_score';
/* global db Dexie */
import { db } from '../util/database.js';
import Dexie from 'dexie';
// importScripts('../../ext/xregexp/nonLetterRegex.js')
var nonLetterRegex =
  /[^\s0-9A-Za-z\xAA\xB5\xBA\xC0-\xD6\xD8-\xF6\xF8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0561-\u0587\u05D0-\u05EA\u05F0-\u05F2\u0620-\u064A\u066E\u066F\u0671-\u06D3\u06D5\u06E5\u06E6\u06EE\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07CA-\u07EA\u07F4\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u08A0-\u08B4\u0904-\u0939\u093D\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098C\u098F\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC\u09DD\u09DF-\u09E1\u09F0\u09F1\u0A05-\u0A0A\u0A0F\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32\u0A33\u0A35\u0A36\u0A38\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0\u0AE1\u0AF9\u0B05-\u0B0C\u0B0F\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32\u0B33\u0B35-\u0B39\u0B3D\u0B5C\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99\u0B9A\u0B9C\u0B9E\u0B9F\u0BA3\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C60\u0C61\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDE\u0CE0\u0CE1\u0CF1\u0CF2\u0D05-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D5F-\u0D61\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32\u0E33\u0E40-\u0E46\u0E81\u0E82\u0E84\u0E87\u0E88\u0E8A\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAA\u0EAB\u0EAD-\u0EB0\u0EB2\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDF\u0F00\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F\u1050-\u1055\u105A-\u105D\u1061\u1065\u1066\u106E-\u1070\u1075-\u1081\u108E\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16F1-\u16F8\u1700-\u170C\u170E-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1877\u1880-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u1A00-\u1A16\u1A20-\u1A54\u1AA7\u1B05-\u1B33\u1B45-\u1B4B\u1B83-\u1BA0\u1BAE\u1BAF\u1BBA-\u1BE5\u1C00-\u1C23\u1C4D-\u1C4F\u1C5A-\u1C7D\u1CE9-\u1CEC\u1CEE-\u1CF1\u1CF5\u1CF6\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2183\u2184\u2C00-\u2C2E\u2C30-\u2C5E\u2C60-\u2CE4\u2CEB-\u2CEE\u2CF2\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005\u3006\u3031-\u3035\u303B\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312D\u3131-\u318E\u31A0-\u31BA\u31F0-\u31FF\u3400-\u4DB5\u4E00-\u9FD5\uA000-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6E5\uA717-\uA71F\uA722-\uA788\uA78B-\uA7AD\uA7B0-\uA7B7\uA7F7-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA840-\uA873\uA882-\uA8B3\uA8F2-\uA8F7\uA8FB\uA8FD\uA90A-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF\uA9E0-\uA9E4\uA9E6-\uA9EF\uA9FA-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB65\uAB70-\uABE2\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40\uFB41\uFB43\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC]/g;
const whitespaceRegex = /\s+/g;

const ignoredCharactersRegex = /[']+/g;

// stop words list from https://github.com/weixsong/elasticlunr.js/blob/master/lib/stop_word_filter.js
const stopWords = {
  '': true,
  a: true,
  able: true,
  about: true,
  across: true,
  after: true,
  all: true,
  almost: true,
  also: true,
  am: true,
  among: true,
  an: true,
  and: true,
  any: true,
  are: true,
  as: true,
  at: true,
  be: true,
  because: true,
  been: true,
  but: true,
  by: true,
  can: true,
  cannot: true,
  could: true,
  dear: true,
  did: true,
  do: true,
  does: true,
  either: true,
  else: true,
  ever: true,
  every: true,
  for: true,
  from: true,
  get: true,
  got: true,
  had: true,
  has: true,
  have: true,
  he: true,
  her: true,
  hers: true,
  him: true,
  his: true,
  how: true,
  however: true,
  i: true,
  if: true,
  in: true,
  into: true,
  is: true,
  it: true,
  its: true,
  just: true,
  least: true,
  let: true,
  like: true,
  likely: true,
  may: true,
  me: true,
  might: true,
  most: true,
  must: true,
  my: true,
  neither: true,
  no: true,
  nor: true,
  not: true,
  of: true,
  off: true,
  often: true,
  on: true,
  only: true,
  or: true,
  other: true,
  our: true,
  own: true,
  rather: true,
  said: true,
  say: true,
  says: true,
  she: true,
  should: true,
  since: true,
  so: true,
  some: true,
  than: true,
  that: true,
  the: true,
  their: true,
  them: true,
  then: true,
  there: true,
  these: true,
  they: true,
  this: true,
  tis: true,
  to: true,
  too: true,
  twas: true,
  us: true,
  wants: true,
  was: true,
  we: true,
  were: true,
  what: true,
  when: true,
  where: true,
  which: true,
  while: true,
  who: true,
  whom: true,
  why: true,
  will: true,
  with: true,
  would: true,
  yet: true,
  you: true,
  your: true,
};

/* this is used in placesWorker.js when a history item is created */
export function tokenize(string) {
  return (
    string
      .trim()
      .toLowerCase()
      .replace(ignoredCharactersRegex, '')
      .replace(nonLetterRegex, ' ')
      // remove diacritics
      // https://stackoverflow.com/a/37511463
      .normalize('NFD')
      .replace(/[\u0300-\u036f]/g, '')
      .split(whitespaceRegex)
      .filter(function (token) {
        return !stopWords[token] && token.length <= 100;
      })
      .slice(0, 20000)
  );
}

// finds the documents that contain all of the prefixes in their searchIndex
// code from https://github.com/dfahlander/Dexie.js/issues/281
function getMatchingDocs(tokens) {
  return db.transaction('r', db.places, function* () {
    // Parallell search for all tokens - just select resulting primary keys
    const tokenMatches = yield Dexie.Promise.all(
      tokens.map((prefix) =>
        db.places.where('searchIndex').equals(prefix).primaryKeys()
      )
    );

    var results = [];

    /*
    A document matches if each search token is either 1) contained in the title, URL, or tags,
    even if it's part of a larger word, or 2) a word in the full-text index.
     */
    historyInMemoryCache.forEach(function (item) {
      var itext = (
        item.url +
        ' ' +
        item.title +
        ' ' +
        item.tags.join(' ')
      ).toLowerCase();
      var matched = true;
      for (var i = 0; i < tokens.length; i++) {
        if (!tokenMatches[i].includes(item.id) && !itext.includes(tokens[i])) {
          matched = false;
          break;
        }
      }
      if (matched) {
        results.push(item);
      }
    });

    /* Finally select entire documents from intersection.
    To improve perf, we only read the full text of 100 documents for ranking,
     but this could mean we miss relevant documents. So sort them based on page
     score (recency + visit count) and then only read the 100 highest-ranking ones,
     since these are most likely to be in the final results.
    */
    const ordered = results
      .sort(function (a, b) {
        return calculateHistoryScore(b) - calculateHistoryScore(a);
      })
      .map((i) => i.id)
      .slice(0, 100);

    return yield db.places.where('id').anyOf(ordered).toArray();
  });
}

export function fullTextPlacesSearch(searchText, callback) {
  const searchWords = tokenize(searchText);
  const sl = searchWords.length;

  if (searchWords.length === 0) {
    callback([]);
    return;
  }

  getMatchingDocs(searchWords).then(function (docs) {
    const totalCounts = {};
    for (let i = 0; i < sl; i++) {
      totalCounts[searchWords[i]] = 0;
    }

    const docTermCounts = {};
    const docIndexes = {};
    let totalIndexLength = 0;

    // find the number and position of the search terms in each document
    docs.forEach(function (doc) {
      const termCount = {};
      const index = doc.searchIndex;
      const indexList = [];

      for (let i = 0; i < sl; i++) {
        let count = 0;
        const token = searchWords[i];

        let idx = doc.searchIndex.indexOf(token);

        while (idx !== -1) {
          count++;
          indexList.push(idx);
          idx = doc.searchIndex.indexOf(token, idx + 1);
        }

        termCount[searchWords[i]] = count;
        totalCounts[searchWords[i]] += count;
      }

      docTermCounts[doc.url] = termCount;
      docIndexes[doc.url] = indexList.sort((a, b) => a - b);
      totalIndexLength += index.length;
    });

    const dl = docs.length;

    for (let i = 0; i < dl; i++) {
      let doc = docs[i];
      const indexLen = doc.searchIndex.length;
      const termCounts = docTermCounts[doc.url];

      if (!doc.boost) {
        doc.boost = 0;
      }

      // add boost when search terms appear more frequently than in other documents
      for (let x = 0; x < sl; x++) {
        doc.boost += Math.min(
          (termCounts[searchWords[x]] /
            indexLen /
            (totalCounts[searchWords[x]] / totalIndexLength)) *
            0.33,
          1
        );
      }

      // add boost when search terms appear close to each other

      const indexList = docIndexes[doc.url];
      let totalWordDistanceBoost = 0;

      for (let n = 1; n < indexList.length; n++) {
        let distance = indexList[n] - indexList[n - 1];
        if (distance < 50) {
          totalWordDistanceBoost += Math.pow(50 - distance, 2) * 0.00005;
        }
        if (distance === 1) {
          totalWordDistanceBoost += 0.04;
        }
      }

      doc.boost += Math.min(totalWordDistanceBoost, 5);

      // these properties are never used, and sending them from the worker takes a long time

      delete doc.pageHTML;
      delete doc.extractedText;
      delete doc.searchIndex;
    }

    callback(docs);
  });
}

function calculateHistoryScore(item) {
  // item.boost - how much the score should be multiplied by. Example - 0.05
  let fs = item.lastVisit * (1 + 0.036 * Math.sqrt(item.visitCount));

  // bonus for short url's
  if (item.url.length < 20) {
    fs += (30 - item.url.length) * 2500;
  }

  if (item.boost) {
    fs += fs * item.boost;
  }

  return fs;
}

const oneDayInMS = 24 * 60 * 60 * 1000; // one day in milliseconds
const oneWeekAgo = Date.now() - oneDayInMS * 7;

// the oldest an item can be to remain in the database
const maxItemAge = oneDayInMS * 42;

function cleanupHistoryDatabase() {
  // removes old history entries
  db.places
    .where('lastVisit')
    .below(Date.now() - maxItemAge)
    .and(function (item) {
      return item.isBookmarked === false;
    })
    .delete();
}

setTimeout(cleanupHistoryDatabase, 20000); // don't run immediately on startup, since is might slow down searchbar search.
setInterval(cleanupHistoryDatabase, 60 * 60 * 1000);

// cache history in memory for faster searching. This actually takes up very little space, so we can cache everything.
let historyInMemoryCache = [];

const spacesRegex = /[\+\s._/-]+/g; // things that could be considered spaces
var tagIndex = {
  totalDocs: 0,
  termDocCounts: {},
  termTags: {},
  tagTagMap: {},
  tagCounts: {},
  tagUpdateTimes: {},
  getPageTokens: function (page) {
    var urlChunk = '';
    try {
      // ignore the TLD, since it doesn't predict tags very well
      urlChunk = new URL(page.url).hostname.split('.').slice(0, -1).join('.');
    } catch (e) {}
    var tokens = tokenize(page.title + ' ' + urlChunk);

    var generic = ['www', 'com', 'net', 'html', 'pdf', 'file'];
    tokens = tokens.filter((t) => t.length > 2 && !generic.includes(t));

    return tokens;
  },
  addPage: function (page) {
    if (page.tags.length === 0) {
      return;
    }

    tagIndex.totalDocs++;

    var tokens = tagIndex.getPageTokens(page);

    tokens
      .filter((t, i) => tokens.indexOf(t) === i)
      .forEach(function (token) {
        if (!tagIndex.termDocCounts[token]) {
          tagIndex.termDocCounts[token] = 1;
        } else {
          tagIndex.termDocCounts[token]++;
        }
      });

    page.tags.forEach(function (tag) {
      tokens.forEach(function (token) {
        if (!tagIndex.termTags[token]) {
          tagIndex.termTags[token] = {};
        }
        if (tagIndex.termTags[token][tag]) {
          tagIndex.termTags[token][tag]++;
        } else {
          tagIndex.termTags[token][tag] = 1;
        }
      });
    });

    page.tags.forEach(function (t1) {
      if (!tagIndex.tagCounts[t1]) {
        tagIndex.tagCounts[t1] = 1;
      } else {
        tagIndex.tagCounts[t1]++;
      }
      page.tags.forEach(function (t2) {
        if (t1 === t2) {
          return;
        }
        if (!tagIndex.tagTagMap[t1]) {
          tagIndex.tagTagMap[t1] = {};
        }

        if (!tagIndex.tagTagMap[t1][t2]) {
          tagIndex.tagTagMap[t1][t2] = 1;
        } else {
          tagIndex.tagTagMap[t1][t2]++;
        }
      });
    });

    page.tags.forEach(function (tag) {
      if (
        !tagIndex.tagUpdateTimes[tag] ||
        page.lastVisit > tagIndex.tagUpdateTimes[tag]
      ) {
        tagIndex.tagUpdateTimes[tag] = page.lastVisit;
      }
    });
  },
  removePage: function (page) {
    if (page.tags.length === 0) {
      return;
    }

    tagIndex.totalDocs--;

    var tokens = tagIndex.getPageTokens(page);

    tokens
      .filter((t, i) => tokens.indexOf(t) === i)
      .forEach(function (token) {
        if (tagIndex.termDocCounts[token]) {
          tagIndex.termDocCounts[token]--;
        }
      });

    page.tags.forEach(function (tag) {
      tokens.forEach(function (token) {
        if (tagIndex.termTags[token] && tagIndex.termTags[token][tag]) {
          tagIndex.termTags[token][tag]--;
        }
      });
    });

    page.tags.forEach(function (t1) {
      if (tagIndex.tagCounts[t1]) {
        tagIndex.tagCounts[t1]--;
      }

      page.tags.forEach(function (t2) {
        if (t1 === t2) {
          return;
        }
        if (!tagIndex.tagTagMap[t1]) {
          tagIndex.tagTagMap[t1] = {};
        }

        if (tagIndex.tagTagMap[t1] && tagIndex.tagTagMap[t1][t2]) {
          tagIndex.tagTagMap[t1][t2]--;
        }
      });
    });
  },
  onChange: function (oldPage, newPage) {
    tagIndex.removePage(oldPage);
    tagIndex.addPage(newPage);
  },
  getAllTagsRanked: function (page) {
    var tokens = tagIndex.getPageTokens(page);
    // get term frequency
    var terms = {};
    tokens.forEach(function (t) {
      if (!terms[t]) {
        terms[t] = 1;
      } else {
        terms[t]++;
      }
    });

    var probs = {};

    for (var term in terms) {
      var tf = terms[term] / tokens.length;
      var idf = Math.log(
        tagIndex.totalDocs / (tagIndex.termDocCounts[term] || 1)
      );
      var tfidf = tf * idf;

      if (tagIndex.termTags[term]) {
        for (var tag in tagIndex.termTags[term]) {
          if (tagIndex.tagCounts[tag] < 2) {
            continue;
          }
          if (!probs[tag]) {
            probs[tag] = 0;
          }
          probs[tag] +=
            (tagIndex.termTags[term][tag] / tagIndex.tagCounts[tag]) * tfidf;
        }
      }
    }

    var probsArr = Object.keys(tagIndex.tagCounts).map((key) => {
      return { tag: key, value: probs[key] || 0 };
    });

    probsArr = probsArr.sort((a, b) => {
      return b.value - a.value;
    });

    return probsArr;
  },
  getSuggestedTags: function (page) {
    return tagIndex
      .getAllTagsRanked(page)
      .filter((p) => p.value > 0.2)
      .map((p) => p.tag);
  },
  getSuggestedItemsForTags: function (tags) {
    var set = historyInMemoryCache
      .filter((i) => i.isBookmarked)
      .map((p) => {
        return { page: p, tags: tagIndex.getSuggestedTags(p) };
      });

    set = set.filter(function (result) {
      for (var i = 0; i < tags.length; i++) {
        if (!result.tags.includes(tags[i])) {
          return false;
        }
      }
      return true;
    });

    return set.map((i) => i.page).slice(0, 50);
  },
  autocompleteTags: function (searchTags) {
    // find which tags are most frequently associated with the searched tags
    var tagScores = [];

    for (var tag in tagIndex.tagCounts) {
      var score = tagIndex.tagCounts[tag];
      searchTags.forEach(function (searchTag) {
        // tagtagMap[searchTag][tag] holds the number of items that have both searchTag and tag
        if (tagIndex.tagTagMap[searchTag]) {
          score *= tagIndex.tagTagMap[searchTag][tag] || 0;
        } else {
          score = 0;
        }
      });

      // prefer tags with a recently-visited (or created) page
      score *= Math.max(
        2 -
          (Date.now() - tagIndex.tagUpdateTimes[tag]) /
            (14 * 24 * 60 * 60 * 1000),
        1
      );

      tagScores.push({ tag, score });
    }

    return tagScores
      .filter((t) => t.score > 0)
      .sort((a, b) => b.score - a.score)
      .map((i) => i.tag);
  },
};
function processSearchText(text) {
  // the order of these transformations is important - for example, spacesRegex removes / characters, so protocols must be removed before it runs
  return text
    .toLowerCase()
    .split('?')[0]
    .replace('http://', '')
    .replace('https://', '')
    .replace('www.', '')
    .replace(spacesRegex, ' ');
}

function searchPlaces(searchText, callback, options) {
  function processSearchItem(item) {
    if (limitToBookmarks && !item.isBookmarked) {
      return;
    }
    let itextURL = processSearchText(item.url);
    let itext = itextURL;

    if (item.url !== item.title) {
      itext += ' ' + item.title.toLowerCase().replace(spacesRegex, ' ');
    }

    if (item.tags) {
      itext += ' ' + item.tags.join(' ');
    }

    const tindex = itext.indexOf(st);

    // if the url contains the search string, count as a match
    // prioritize matches near the beginning of the url
    if (tindex === 0) {
      item.boost = itemStartBoost; // large amount of boost for this
      matches.push(item);
    } else if (tindex !== -1) {
      item.boost = exactMatchBoost;
      matches.push(item);
    } else {
      // if all of the search words (split by spaces, etc) exist in the url, count it as a match, even if they are out of order

      if (substringSearchEnabled) {
        let substringMatch = true;

        // check if the search text matches but is out of order
        for (let i = 0; i < swl; i++) {
          if (itext.indexOf(searchWords[i]) === -1) {
            substringMatch = false;
            break;
          }
        }

        if (substringMatch) {
          item.boost = 0.125 * swl + 0.02 * stl;
          matches.push(item);
          return;
        }
      }

      if (item.visitCount !== 1 || item.lastVisit > oneWeekAgo) {
        // if the item has been visited more than once, or has been visited in the last week, we should calculate the fuzzy score. Otherwise, it is ignored. This reduces the number of bad results and increases search speed.
        const score = itextURL.score(st, 0);

        if (score > 0.4 + 0.00075 * itextURL.length) {
          item.boost = score * 0.5;

          if (score > 0.62) {
            item.boost += 0.33;
          }

          matches.push(item);
        }
      }
    }
  }

  const matches = [];
  const st = processSearchText(searchText);
  const stl = searchText.length;
  const searchWords = st.split(' ');
  const swl = searchWords.length;
  let substringSearchEnabled = false;
  const itemStartBoost = Math.min(2.5 * stl, 10);
  const exactMatchBoost = 0.4 + 0.075 * stl;
  const limitToBookmarks = options && options.searchBookmarks;
  const resultsLimit = (options && options.limit) || 100;

  if (searchText.indexOf(' ') !== -1) {
    substringSearchEnabled = true;
  }

  for (let i = 0; i < historyInMemoryCache.length; i++) {
    if (matches.length > resultsLimit * 2) {
      break;
    }
    processSearchItem(historyInMemoryCache[i]);
  }

  matches.sort(function (a, b) {
    // we have to re-sort to account for the boosts applied to the items
    return calculateHistoryScore(b) - calculateHistoryScore(a);
  });

  // clean up
  matches.forEach(function (match) {
    match.boost = 0;
  });

  callback(matches.slice(0, resultsLimit));
}

function addToHistoryCache(item) {
  if (item.isBookmarked) {
    tagIndex.addPage(item);
  }
  delete item.pageHTML;
  delete item.extractedText;
  delete item.searchIndex;

  historyInMemoryCache.push(item);
}

function addOrUpdateHistoryCache(item) {
  delete item.pageHTML;
  delete item.extractedText;
  delete item.searchIndex;

  let oldItem;

  for (let i = 0; i < historyInMemoryCache.length; i++) {
    if (historyInMemoryCache[i].url === item.url) {
      oldItem = historyInMemoryCache[i];
      historyInMemoryCache[i] = item;
      break;
    }
  }

  if (!oldItem) {
    historyInMemoryCache.push(item);
  }

  if (oldItem) {
    tagIndex.onChange(oldItem, item);
  }
}

function loadHistoryInMemory() {
  historyInMemoryCache = [];

  db.places
    .orderBy('visitCount')
    .reverse()
    .each(function (item) {
      addToHistoryCache(item);
    })
    .then(function () {
      // if we have enough matches during the search, we exit. In order for this to work, frequently visited sites have to come first in the cache.
      historyInMemoryCache.sort(function (a, b) {
        return calculateHistoryScore(b) - calculateHistoryScore(a);
      });
    });
}

loadHistoryInMemory();

onmessage = function (e) {
  const action = e.data.action;
  const pageData = e.data.pageData;
  const flags = e.data.flags || {};
  const searchText = e.data.text && e.data.text.toLowerCase();
  const callbackId = e.data.callbackId;
  const options = e.data.options;

  if (action === 'updatePlace') {
    db.transaction('rw', db.places, function () {
      db.places
        .where('url')
        .equals(pageData.url)
        .first(function (item) {
          var isNewItem = false;
          if (!item) {
            isNewItem = true;
            item = {
              url: pageData.url,
              title: pageData.url,
              color: null,
              visitCount: 0,
              lastVisit: Date.now(),
              pageHTML: '',
              extractedText: '',
              searchIndex: [],
              isBookmarked: false,
              tags: [],
              metadata: {},
            };
          }
          for (const key in pageData) {
            if (key === 'extractedText') {
              item.searchIndex = tokenize(pageData.extractedText);
            } else {
              item[key] = pageData[key];
            }
          }

          if (flags.isNewVisit) {
            item.visitCount++;
            item.lastVisit = Date.now();
          }
          // console.log("places worker===========");
          // console.log(item)
          db.places.put(item);
          if (isNewItem) {
            addToHistoryCache(item);
          } else {
            addOrUpdateHistoryCache(item);
          }
          postMessage({
            result: null,
            callbackId: callbackId,
          });
        })
        .catch(function (err) {
          console.warn('failed to update history.');
          console.warn('page url was: ' + pageData.url);
          console.error(err);
        });
    });
  }

  if (action === 'deleteHistory') {
    db.places.where('url').equals(pageData.url).delete();

    // delete from the in-memory cache
    for (let i = 0; i < historyInMemoryCache.length; i++) {
      if (historyInMemoryCache[i].url === pageData.url) {
        historyInMemoryCache.splice(i, 1);
      }
    }
  }

  if (action === 'deleteAllHistory') {
    db.places
      .filter(function (item) {
        return item.isBookmarked === false;
      })
      .delete()
      .then(function () {
        loadHistoryInMemory();
      });
  }

  if (action === 'getSuggestedTags') {
    postMessage({
      result: tagIndex.getSuggestedTags(
        historyInMemoryCache.find((i) => i.url === pageData.url)
      ),
      callbackId: callbackId,
    });
  }

  if (action === 'getAllTagsRanked') {
    postMessage({
      result: tagIndex.getAllTagsRanked(
        historyInMemoryCache.find((i) => i.url === pageData.url)
      ),
      callbackId: callbackId,
    });
  }

  if (action === 'getSuggestedItemsForTags') {
    postMessage({
      result: tagIndex.getSuggestedItemsForTags(pageData.tags),
      callbackId: callbackId,
    });
  }

  if (action === 'autocompleteTags') {
    postMessage({
      result: tagIndex.autocompleteTags(pageData.tags),
      callbackId: callbackId,
    });
  }

  if (action === 'searchPlaces') {
    // do a history search
    searchPlaces(
      searchText,
      function (matches) {
        postMessage({
          result: matches,
          callbackId: callbackId,
        });
      },
      options
    );
  }

  if (action === 'searchPlacesFullText') {
    fullTextPlacesSearch(searchText, function (matches) {
      matches.sort(function (a, b) {
        return calculateHistoryScore(b) - calculateHistoryScore(a);
      });

      postMessage({
        result: matches.slice(0, 100),
        callbackId: callbackId,
      });
    });
  }

  if (action === 'getPlaceSuggestions') {
    // get the history item for the provided url

    let baseItem = null;

    for (let i = 0; i < historyInMemoryCache.length; i++) {
      if (historyInMemoryCache[i].url === searchText) {
        baseItem = historyInMemoryCache[i];
        break;
      }
    }

    // use a default item. This could occur if the given url is for a page that has never finished loading
    if (!baseItem) {
      baseItem = {
        url: '',
        title: '',
        lastVisit: Date.now(),
        visitCount: 1,
      };
    }

    const cTime = Date.now();

    let results = historyInMemoryCache
      .slice()
      .filter((i) => cTime - i.lastVisit < 604800000);

    for (let i = 0; i < results.length; i++) {
      results[i].hScore = calculateHistoryScore(results[i]);
    }

    results = results.sort(function (a, b) {
      return b.hScore - a.hScore;
    });

    postMessage({
      result: results.slice(0, 100),
      callbackId: callbackId,
    });
  }
};
